# -*- coding: utf-8 -*-

import datetime
import logging 

import webapp2
from google.appengine.ext import blobstore
from google.appengine.api import files
    
class BuildKoreanTwitterTrends(webapp2.RequestHandler):
    _stopwordDictionary = {} # {stopword: True or False}
    _punctuationList = ['RT', '&lt;', '&gt;', '&amp;', '\"', '\'', '(', ')', '[', ']', ',', '!', '?', '.']
            
    _userDictionary = {}
    _hashtagDictionary = {}
    _linkDictionary = {}
    _wordDictionary = {}

    _basetime = None
    _hours = 3
    _supportThreshold = 10
    _host = 'koreantrends' # if host is changed to 'dev', then codes in development will be executed.
    
    def _store(self):
        '''
        temporary code
        '''
        if self._host == 'dev':
            fileName = 'test_koreantrends'
        else:
            fileName = 'koreantrends'

        maxRetry = 3
        errorCount = 0
        while errorCount < maxRetry:
            try:
                blobName = files.blobstore.create(mime_type = 'text/plain', _blobinfo_uploaded_filename = fileName)
                break
            except Exception, e:
                logging.error('Failed to open ' + fileName)
                logging.error(e)
                errorCount += 1
        if errorCount == maxRetry:
            logging.error('Tried %d times to open %s, but failed' % (maxRetry, fileName))
            return
            
        userList = self._userDictionary.keys()
        for user in userList:
            with files.open(blobName, 'a') as f:
                if len(self._userDictionary[user]) >= self._supportThreshold:
                    f.write(user + '\t' + str(self._userDictionary[user]) + '\n')
            f.close()
        
        hashtagList = self._hashtagDictionary.keys()
        for hashtag in hashtagList:
            with files.open(blobName, 'a') as f:
                if len(self._hashtagDictionary[hashtag]) >= self._supportThreshold:
                    f.write(hashtag + '\t' + str(self._hashtagDictionary[hashtag]) + '\n')
            f.close()
    
        linkList = self._linkDictionary.keys()
        for link in linkList:
            with files.open(blobName, 'a') as f:
                if len(self._linkDictionary[link]) >= self._supportThreshold:
                    f.write(link + '\t' + str(self._linkDictionary[link]) + '\n')
            f.close()
        
        wordList = self._wordDictionary.keys()
        index = 0
        wordListLength = len(wordList)
        while index < wordListLength:
            with files.open(blobName, 'a') as f:
                # Blobfiles on GAE can be opened just for 30 seconds, so repeat to reopen
                count = 0
                for word in wordList[index:]:
                    if len(self._wordDictionary[word]) >= self._supportThreshold:
                        f.write(word + '\t' + str(self._wordDictionary[word]) + '\n')
                    count += 1
                    if count == 1000: break
            f.close()
            index += count
                    
        files.finalize(blobName)        
        
        return

    def _removePunctuationMarks(self, text):
        '''
        remove some punctuation marks.
        '''
        # Be cautious! ':', '/', and '.' are used in http links,
        # and '#' is used to indicate hash tags.
        for punctuation in self._punctuationList:
            text = text.replace(punctuation, ' ')
           
        return text

    def _build(self):
        '''
        open blobstore files and build word dictionaries whose key is a Korean word and value is its index.
        '''
        # open files generated from self._hours hours ago till now
        query = blobstore.BlobInfo.gql("WHERE filename = :1 AND creation >= :2 AND creation < :3 ORDER BY creation", 
                                       'korean_twitter_sample',
                                       datetime.datetime.strptime(self._basetime, '%Y%m%d%H') - datetime.timedelta(hours = self._hours),
                                       datetime.datetime.strptime(self._basetime, '%Y%m%d%H'))
        for result in query.run():
            blobFile = result.open()
            for line in blobFile:
                index, tweet = line.split('\t')
                tweet = unicode(tweet, 'utf-8')
                for candidate in tweet.split():
                    if len(candidate) == 0:
                        continue
                    elif candidate[0] == '@': # Twitter user
                        if candidate[-1] == ':':
                            self._userDictionary[candidate[:-1]] = self._userDictionary.get(candidate[:-1], []) + [index]
                        else: 
                            self._userDictionary[candidate] = self._userDictionary.get(candidate, []) + [index]
                        continue
                    elif candidate[0] == '#': # hashtag
                        self._hashtagDictionary[candidate] = self._hashtagDictionary.get(candidate, []) + [index]
                        continue
                    elif candidate[:7] == 'http://': # http link
                        self._linkDictionary[candidate] = self._linkDictionary.get(candidate, []) + [index]
                        continue
                    else: # plain words
                        for word in self._removePunctuationMarks(candidate).split():
                            if not self._stopwordDictionary.get(word, False):
                                self._wordDictionary[word] = self._wordDictionary.get(word, []) + [index]
            blobFile.close()
        
        self._store()
            
        return
    
    def get(self):
        '''
        Analyzing blobstore files saved by FetchKoreanTwitterSample, build Korean Twitter trends 
        '''
        self.response.headers['Content-Type'] = 'text/plain'
        
        fullDomain = self.request.headers['Host']
        if fullDomain.find('dev.') != -1 or fullDomain.find('localhost:') != -1:
            self._host = 'dev'
            logging.debug('Host: ' + self._host)

        # a parameter 'basetime' whose format is '%Y%m%d%H' and default is the last hour.
        if self.request.get('basetime'):
            self._basetime = self.request.get('basetime')
        else:
            self._basetime = datetime.datetime.now()
            self._basetime = self._basetime.strftime('%Y%m%d%H')
        logging.debug('basetime: ' + self._basetime)
            
        self._build()
        
        return

        
app = webapp2.WSGIApplication([('/build_korean_twitter_trends', BuildKoreanTwitterTrends)], debug = True)
